{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "accelerator": "GPU",
    "colab": {
      "name": "fer2013.ipynb",
      "provenance": [],
      "collapsed_sections": [],
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/amilkh/cs230-fer/blob/master/baseline-model.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "gwdg7Sv3XBaP",
        "outputId": "45847514-a0cd-425e-ad4e-f827c0b090d8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "%tensorflow_version 1.4"
      ],
      "execution_count": 1,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "`%tensorflow_version` only switches the major version: 1.x or 2.x.\n",
            "You set: `1.4`. This will be interpreted as: `1.x`.\n",
            "\n",
            "\n",
            "TensorFlow 1.x selected.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "colab_type": "code",
        "id": "2nz38mJZXN_P",
        "outputId": "13f963d4-e549-4924-f8b4-b944fe985213",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "import matplotlib.pyplot as plt\n",
        "import pandas as pd\n",
        "import numpy as np\n",
        "\n",
        "import tensorflow as tf\n",
        "from tensorflow.python.lib.io import file_io\n",
        "\n",
        "import keras\n",
        "from keras.preprocessing.image import ImageDataGenerator\n",
        "\n",
        "from keras.models import Sequential\n",
        "from keras.layers import Dense, Dropout, Flatten\n",
        "from keras.layers import Conv2D, MaxPooling2D, BatchNormalization\n",
        "from keras.optimizers import SGD\n",
        "from keras.callbacks import ReduceLROnPlateau\n",
        "from keras.callbacks import ModelCheckpoint, EarlyStopping\n",
        "\n",
        "from sklearn.metrics import confusion_matrix\n",
        "from seaborn import heatmap\n",
        "\n",
        "%matplotlib inline"
      ],
      "execution_count": 2,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Using TensorFlow backend.\n"
          ],
          "name": "stderr"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "KI204HIYEdLE",
        "colab_type": "code",
        "outputId": "162c90e1-0d1f-4138-e41d-f95b30ff80ae",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 34
        }
      },
      "source": [
        "from google.colab import drive\n",
        "drive.mount('/content/drive')"
      ],
      "execution_count": 3,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "Drive already mounted at /content/drive; to attempt to forcibly remount, call drive.mount(\"/content/drive\", force_remount=True).\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "vqF6jK_u5Q7d",
        "colab_type": "code",
        "outputId": "e096abf6-cc77-47ba-bfbf-0cfb8ccfc35b",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 51
        }
      },
      "source": [
        "print(tf.__version__)\n",
        "print(keras.__version__)"
      ],
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "1.15.0\n",
            "2.2.5\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "oPO33wZKzHsc",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "EPOCHS = 150\n",
        "BS = 128\n",
        "DROPOUT_RATE = 0.3\n",
        "SGD_LEARNING_RATE = 0.01\n",
        "SGD_DECAY = 0.0001"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "bG4t-6RO-g2S",
        "colab_type": "code",
        "outputId": "5d7d2d45-5ee2-40e1-d99c-0e6117a98baf",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 564
        }
      },
      "source": [
        "model = Sequential()\n",
        "model.add(BatchNormalization(input_shape=(48,48,1)))\n",
        "model.add(Conv2D(32, (3, 3), activation='relu',padding='same', input_shape=(48,48,1),name=\"conv1\"))\n",
        "model.add(BatchNormalization())\n",
        "#model.add(MaxPooling2D(pool_size=(2, 2),name=\"maxpool1\"))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Conv2D(32, (3, 3), activation='relu',padding='same',name=\"conv2\"))\n",
        "model.add(BatchNormalization())\n",
        "model.add(MaxPooling2D(pool_size=(2, 2),name=\"maxpool2\"))         \n",
        "model.add(Dropout(0.2))\n",
        "model.add(Conv2D(64, (3, 3), activation='relu',padding='same',name=\"conv3\"))\n",
        "model.add(BatchNormalization())\n",
        "model.add(MaxPooling2D(pool_size=(2, 2),name=\"maxpool3\"))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Conv2D(64, (3, 3), activation='relu',padding='same',name=\"conv4\"))\n",
        "model.add(BatchNormalization())\n",
        "model.add(MaxPooling2D(pool_size=(2, 2),name=\"maxpool4\"))\n",
        "model.add(Dropout(0.2))\n",
        "model.add(Flatten())\n",
        "model.add(Dense(1024, activation='relu',name='fc1'))\n",
        "model.add(Dropout(DROPOUT_RATE))\n",
        "model.add(BatchNormalization())\n",
        "model.add(Dense(7, activation='softmax',name='fcsoftmax'))\n",
        "\n",
        "#TODO: weight decay of 0.0001...initial learning rate is set to 0.01 and reduced by a factor of 2 at every 25 epoch\n",
        "sgd = SGD(lr=SGD_LEARNING_RATE,momentum=0.9, decay=SGD_DECAY, nesterov=True)\n",
        "model.compile(loss='categorical_crossentropy',optimizer=sgd,metrics=['accuracy'])\n",
        "\n",
        "# checkpoint\n",
        "cp_filepath='/content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5'\n",
        "rlrop = keras.callbacks.ReduceLROnPlateau(monitor='val_acc',mode='max',factor=0.5, patience=10, min_lr=0.00001, verbose=1)\n",
        "checkpoint = ModelCheckpoint(cp_filepath, monitor='val_acc', verbose=1, save_best_only=True, mode='max')\n",
        "callbacks_list = [rlrop,checkpoint]"
      ],
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:66: The name tf.get_default_graph is deprecated. Please use tf.compat.v1.get_default_graph instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:541: The name tf.placeholder is deprecated. Please use tf.compat.v1.placeholder instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:190: The name tf.get_default_session is deprecated. Please use tf.compat.v1.get_default_session instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:197: The name tf.ConfigProto is deprecated. Please use tf.compat.v1.ConfigProto instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:203: The name tf.Session is deprecated. Please use tf.compat.v1.Session instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:207: The name tf.global_variables is deprecated. Please use tf.compat.v1.global_variables instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:216: The name tf.is_variable_initialized is deprecated. Please use tf.compat.v1.is_variable_initialized instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:223: The name tf.variables_initializer is deprecated. Please use tf.compat.v1.variables_initializer instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:2041: The name tf.nn.fused_batch_norm is deprecated. Please use tf.compat.v1.nn.fused_batch_norm instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:148: The name tf.placeholder_with_default is deprecated. Please use tf.compat.v1.placeholder_with_default instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4432: The name tf.random_uniform is deprecated. Please use tf.random.uniform instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3733: calling dropout (from tensorflow.python.ops.nn_ops) with keep_prob is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Please use `rate` instead of `keep_prob`. Rate should be set to `rate = 1 - keep_prob`.\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:4267: The name tf.nn.max_pool is deprecated. Please use tf.nn.max_pool2d instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/optimizers.py:793: The name tf.train.Optimizer is deprecated. Please use tf.compat.v1.train.Optimizer instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:3576: The name tf.log is deprecated. Please use tf.math.log instead.\n",
            "\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ruuR9M08W7Cn",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "! rm -rf train; mkdir train\n",
        "! unzip -q '/content/drive/My Drive/cs230 project/dataset/emotion.zip' -d train\n",
        "! unzip -q '/content/drive/My Drive/cs230 project/dataset/facesdb.zip' -d train\n",
        "! unzip -q '/content/drive/My Drive/cs230 project/dataset/fer2013/train.zip' -d train\n",
        "! unzip -q '/content/drive/My Drive/cs230 project/dataset/googlesearch.zip' -d train\n",
        "! unzip -q '/content/drive/My Drive/cs230 project/dataset/googleset.zip' -d train\n",
        "! unzip -q '/content/drive/My Drive/cs230 project/dataset/jaffe.zip' -d train\n",
        "! unzip -q '/content/drive/My Drive/cs230 project/dataset/umea.zip' -d train\n",
        "! unzip -q '/content/drive/My Drive/cs230 project/dataset/ck-plus.zip' -d train"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "RjjKaFnDW9Kb",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "! rm -rf dev; mkdir dev\n",
        "! unzip -q '/content/drive/My Drive/cs230 project/dataset/fer2013/test-public.zip' -d dev\n",
        "! rm -rf test; mkdir test\n",
        "! unzip -q '/content/drive/My Drive/cs230 project/dataset/fer2013/test-private.zip' -d test"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "7Hgz_lNsW_2z",
        "colab_type": "code",
        "outputId": "d96740f9-5a20-4dc5-aa16-6fe2b3542685",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 136
        }
      },
      "source": [
        "%%bash\n",
        "root='/content/train/'\n",
        "IFS=$(echo -en \"\\n\\b\")\n",
        "(for dir in $(ls -1 \"$root\")\n",
        "    do printf \"$dir: \" && ls -i \"$root$dir\" | wc -l\n",
        " done)"
      ],
      "execution_count": 9,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "0 angry: 4330\n",
            "1 disgust: 1059\n",
            "2 fear: 4372\n",
            "3 happy: 7662\n",
            "4 sad: 5122\n",
            "5 surprise: 3647\n",
            "6 neutral: 5453\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "i9ITyLGoXEC3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from tensorflow.keras.preprocessing.image import ImageDataGenerator\n",
        "\n",
        "def get_datagen(dataset, aug=False):\n",
        "    if aug:\n",
        "        datagen = ImageDataGenerator(\n",
        "                            rescale=1./255,\n",
        "                            featurewise_center=False,\n",
        "                            featurewise_std_normalization=False,\n",
        "                            rotation_range=10,\n",
        "                            width_shift_range=0.1,\n",
        "                            height_shift_range=0.1,\n",
        "                            zoom_range=0.1,\n",
        "                            horizontal_flip=True)\n",
        "    else:\n",
        "        datagen = ImageDataGenerator(rescale=1./255)\n",
        "\n",
        "    return datagen.flow_from_directory(\n",
        "            dataset,\n",
        "            target_size=(48, 48),\n",
        "            color_mode='grayscale',\n",
        "            shuffle = True,\n",
        "            class_mode='categorical',\n",
        "            batch_size=BS)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "k3Mh89rnXXpD",
        "colab_type": "code",
        "outputId": "d0ddf48c-9a79-465d-92c8-b347c7ad3100",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 102
        }
      },
      "source": [
        "train_generator  = get_datagen('/content/train', True)\n",
        "dev_generator    = get_datagen('/content/dev')\n",
        "test_generator  = get_datagen('/content/test')\n"
      ],
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "/usr/local/lib/python3.6/dist-packages/keras_preprocessing/image/utils.py:173: UserWarning: Using \".tiff\" files with multiple bands will cause distortion. Please verify your output.\n",
            "  warnings.warn('Using \".tiff\" files with multiple bands '\n"
          ],
          "name": "stderr"
        },
        {
          "output_type": "stream",
          "text": [
            "Found 31645 images belonging to 7 classes.\n",
            "Found 3589 images belonging to 7 classes.\n",
            "Found 3589 images belonging to 7 classes.\n"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "nTzccX-yXZy5",
        "colab_type": "code",
        "outputId": "8f6fba38-e0c4-4e50-e5d4-c29bae51eb57",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 1000
        }
      },
      "source": [
        "history = model.fit_generator(\n",
        "    generator = train_generator,\n",
        "    validation_data=dev_generator, \n",
        "    #steps_per_epoch=28709// BS,\n",
        "    #validation_steps=3509 // BS,\n",
        "    shuffle=True,\n",
        "    epochs=EPOCHS,\n",
        "    callbacks=callbacks_list,\n",
        "#    callbacks=[rlrop],\n",
        "    use_multiprocessing=False,\n",
        ") "
      ],
      "execution_count": 0,
      "outputs": [
        {
          "output_type": "stream",
          "text": [
            "WARNING:tensorflow:From /tensorflow-1.15.0/python3.6/tensorflow_core/python/ops/math_grad.py:1424: where (from tensorflow.python.ops.array_ops) is deprecated and will be removed in a future version.\n",
            "Instructions for updating:\n",
            "Use tf.where in 2.0, which has the same broadcast rule as np.where\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1033: The name tf.assign_add is deprecated. Please use tf.compat.v1.assign_add instead.\n",
            "\n",
            "WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/keras/backend/tensorflow_backend.py:1020: The name tf.assign is deprecated. Please use tf.compat.v1.assign instead.\n",
            "\n",
            "Epoch 1/150\n",
            "248/248 [==============================] - 33s 132ms/step - loss: 1.9923 - acc: 0.2637 - val_loss: 1.6430 - val_acc: 0.3653\n",
            "\n",
            "Epoch 00001: val_acc improved from -inf to 0.36528, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 2/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 1.7753 - acc: 0.3219 - val_loss: 1.5590 - val_acc: 0.4099\n",
            "\n",
            "Epoch 00002: val_acc improved from 0.36528 to 0.40986, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 3/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.6891 - acc: 0.3553 - val_loss: 1.4688 - val_acc: 0.4313\n",
            "\n",
            "Epoch 00003: val_acc improved from 0.40986 to 0.43132, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 4/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.6090 - acc: 0.3896 - val_loss: 1.3794 - val_acc: 0.4670\n",
            "\n",
            "Epoch 00004: val_acc improved from 0.43132 to 0.46698, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 5/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 1.5502 - acc: 0.4106 - val_loss: 1.3394 - val_acc: 0.4879\n",
            "\n",
            "Epoch 00005: val_acc improved from 0.46698 to 0.48788, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 6/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.5007 - acc: 0.4340 - val_loss: 1.2920 - val_acc: 0.5082\n",
            "\n",
            "Epoch 00006: val_acc improved from 0.48788 to 0.50822, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 7/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.4587 - acc: 0.4462 - val_loss: 1.2809 - val_acc: 0.5107\n",
            "\n",
            "Epoch 00007: val_acc improved from 0.50822 to 0.51073, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 8/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 1.4237 - acc: 0.4621 - val_loss: 1.2659 - val_acc: 0.5180\n",
            "\n",
            "Epoch 00008: val_acc improved from 0.51073 to 0.51797, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 9/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 1.4069 - acc: 0.4653 - val_loss: 1.2378 - val_acc: 0.5375\n",
            "\n",
            "Epoch 00009: val_acc improved from 0.51797 to 0.53748, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 10/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.3724 - acc: 0.4806 - val_loss: 1.2355 - val_acc: 0.5283\n",
            "\n",
            "Epoch 00010: val_acc did not improve from 0.53748\n",
            "Epoch 11/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.3540 - acc: 0.4845 - val_loss: 1.2108 - val_acc: 0.5469\n",
            "\n",
            "Epoch 00011: val_acc improved from 0.53748 to 0.54695, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 12/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.3351 - acc: 0.4966 - val_loss: 1.2145 - val_acc: 0.5444\n",
            "\n",
            "Epoch 00012: val_acc did not improve from 0.54695\n",
            "Epoch 13/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 1.3095 - acc: 0.5037 - val_loss: 1.1764 - val_acc: 0.5559\n",
            "\n",
            "Epoch 00013: val_acc improved from 0.54695 to 0.55587, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 14/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.2926 - acc: 0.5064 - val_loss: 1.1490 - val_acc: 0.5637\n",
            "\n",
            "Epoch 00014: val_acc improved from 0.55587 to 0.56367, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 15/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.2836 - acc: 0.5155 - val_loss: 1.1433 - val_acc: 0.5614\n",
            "\n",
            "Epoch 00015: val_acc did not improve from 0.56367\n",
            "Epoch 16/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.2681 - acc: 0.5193 - val_loss: 1.1254 - val_acc: 0.5759\n",
            "\n",
            "Epoch 00016: val_acc improved from 0.56367 to 0.57593, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 17/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.2558 - acc: 0.5255 - val_loss: 1.1473 - val_acc: 0.5692\n",
            "\n",
            "Epoch 00017: val_acc did not improve from 0.57593\n",
            "Epoch 18/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 1.2440 - acc: 0.5296 - val_loss: 1.1115 - val_acc: 0.5756\n",
            "\n",
            "Epoch 00018: val_acc did not improve from 0.57593\n",
            "Epoch 19/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.2339 - acc: 0.5330 - val_loss: 1.1060 - val_acc: 0.5851\n",
            "\n",
            "Epoch 00019: val_acc improved from 0.57593 to 0.58512, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 20/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.2287 - acc: 0.5350 - val_loss: 1.1083 - val_acc: 0.5871\n",
            "\n",
            "Epoch 00020: val_acc improved from 0.58512 to 0.58707, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 21/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.2084 - acc: 0.5434 - val_loss: 1.1130 - val_acc: 0.5899\n",
            "\n",
            "Epoch 00021: val_acc improved from 0.58707 to 0.58986, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 22/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.2016 - acc: 0.5440 - val_loss: 1.1087 - val_acc: 0.5823\n",
            "\n",
            "Epoch 00022: val_acc did not improve from 0.58986\n",
            "Epoch 23/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.1902 - acc: 0.5478 - val_loss: 1.0864 - val_acc: 0.5926\n",
            "\n",
            "Epoch 00023: val_acc improved from 0.58986 to 0.59264, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 24/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.1856 - acc: 0.5485 - val_loss: 1.0841 - val_acc: 0.5963\n",
            "\n",
            "Epoch 00024: val_acc improved from 0.59264 to 0.59627, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 25/150\n",
            "248/248 [==============================] - 30s 123ms/step - loss: 1.1830 - acc: 0.5541 - val_loss: 1.0917 - val_acc: 0.5868\n",
            "\n",
            "Epoch 00025: val_acc did not improve from 0.59627\n",
            "Epoch 26/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.1706 - acc: 0.5576 - val_loss: 1.0794 - val_acc: 0.5926\n",
            "\n",
            "Epoch 00026: val_acc did not improve from 0.59627\n",
            "Epoch 27/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.1638 - acc: 0.5611 - val_loss: 1.0761 - val_acc: 0.6013\n",
            "\n",
            "Epoch 00027: val_acc improved from 0.59627 to 0.60128, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 28/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.1604 - acc: 0.5621 - val_loss: 1.0887 - val_acc: 0.5974\n",
            "\n",
            "Epoch 00028: val_acc did not improve from 0.60128\n",
            "Epoch 29/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.1510 - acc: 0.5656 - val_loss: 1.0689 - val_acc: 0.5974\n",
            "\n",
            "Epoch 00029: val_acc did not improve from 0.60128\n",
            "Epoch 30/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.1478 - acc: 0.5643 - val_loss: 1.0720 - val_acc: 0.5988\n",
            "\n",
            "Epoch 00030: val_acc did not improve from 0.60128\n",
            "Epoch 31/150\n",
            "248/248 [==============================] - 31s 124ms/step - loss: 1.1397 - acc: 0.5687 - val_loss: 1.0564 - val_acc: 0.6016\n",
            "\n",
            "Epoch 00031: val_acc improved from 0.60128 to 0.60156, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 32/150\n",
            "248/248 [==============================] - 31s 125ms/step - loss: 1.1366 - acc: 0.5710 - val_loss: 1.0718 - val_acc: 0.6088\n",
            "\n",
            "Epoch 00032: val_acc improved from 0.60156 to 0.60880, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 33/150\n",
            "248/248 [==============================] - 31s 125ms/step - loss: 1.1260 - acc: 0.5753 - val_loss: 1.0628 - val_acc: 0.6105\n",
            "\n",
            "Epoch 00033: val_acc improved from 0.60880 to 0.61048, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 34/150\n",
            "248/248 [==============================] - 32s 127ms/step - loss: 1.1210 - acc: 0.5762 - val_loss: 1.0530 - val_acc: 0.5988\n",
            "\n",
            "Epoch 00034: val_acc did not improve from 0.61048\n",
            "Epoch 35/150\n",
            "248/248 [==============================] - 32s 131ms/step - loss: 1.1194 - acc: 0.5790 - val_loss: 1.0643 - val_acc: 0.6063\n",
            "\n",
            "Epoch 00035: val_acc did not improve from 0.61048\n",
            "Epoch 36/150\n",
            "248/248 [==============================] - 31s 124ms/step - loss: 1.1150 - acc: 0.5792 - val_loss: 1.0554 - val_acc: 0.6018\n",
            "\n",
            "Epoch 00036: val_acc did not improve from 0.61048\n",
            "Epoch 37/150\n",
            "248/248 [==============================] - 30s 123ms/step - loss: 1.1105 - acc: 0.5819 - val_loss: 1.0414 - val_acc: 0.6108\n",
            "\n",
            "Epoch 00037: val_acc improved from 0.61048 to 0.61076, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 38/150\n",
            "248/248 [==============================] - 31s 123ms/step - loss: 1.1071 - acc: 0.5844 - val_loss: 1.0478 - val_acc: 0.6130\n",
            "\n",
            "Epoch 00038: val_acc improved from 0.61076 to 0.61298, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 39/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.1036 - acc: 0.5857 - val_loss: 1.0536 - val_acc: 0.6066\n",
            "\n",
            "Epoch 00039: val_acc did not improve from 0.61298\n",
            "Epoch 40/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.0880 - acc: 0.5888 - val_loss: 1.0369 - val_acc: 0.6135\n",
            "\n",
            "Epoch 00040: val_acc improved from 0.61298 to 0.61354, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 41/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.0880 - acc: 0.5873 - val_loss: 1.0327 - val_acc: 0.6239\n",
            "\n",
            "Epoch 00041: val_acc improved from 0.61354 to 0.62385, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 42/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.0904 - acc: 0.5895 - val_loss: 1.0328 - val_acc: 0.6155\n",
            "\n",
            "Epoch 00042: val_acc did not improve from 0.62385\n",
            "Epoch 43/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.0830 - acc: 0.5925 - val_loss: 1.0364 - val_acc: 0.6094\n",
            "\n",
            "Epoch 00043: val_acc did not improve from 0.62385\n",
            "Epoch 44/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.0796 - acc: 0.5932 - val_loss: 1.0348 - val_acc: 0.6183\n",
            "\n",
            "Epoch 00044: val_acc did not improve from 0.62385\n",
            "Epoch 45/150\n",
            "248/248 [==============================] - 31s 123ms/step - loss: 1.0785 - acc: 0.5935 - val_loss: 1.0263 - val_acc: 0.6166\n",
            "\n",
            "Epoch 00045: val_acc did not improve from 0.62385\n",
            "Epoch 46/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.0691 - acc: 0.5998 - val_loss: 1.0318 - val_acc: 0.6188\n",
            "\n",
            "Epoch 00046: val_acc did not improve from 0.62385\n",
            "Epoch 47/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.0652 - acc: 0.5996 - val_loss: 1.0628 - val_acc: 0.6027\n",
            "\n",
            "Epoch 00047: val_acc did not improve from 0.62385\n",
            "Epoch 48/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.0666 - acc: 0.5972 - val_loss: 1.0268 - val_acc: 0.6138\n",
            "\n",
            "Epoch 00048: val_acc did not improve from 0.62385\n",
            "Epoch 49/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.0629 - acc: 0.5986 - val_loss: 1.0141 - val_acc: 0.6213\n",
            "\n",
            "Epoch 00049: val_acc did not improve from 0.62385\n",
            "Epoch 50/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 1.0571 - acc: 0.6032 - val_loss: 1.0206 - val_acc: 0.6222\n",
            "\n",
            "Epoch 00050: val_acc did not improve from 0.62385\n",
            "Epoch 51/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.0553 - acc: 0.6046 - val_loss: 1.0214 - val_acc: 0.6272\n",
            "\n",
            "Epoch 00051: val_acc improved from 0.62385 to 0.62719, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 52/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 1.0559 - acc: 0.6013 - val_loss: 1.0123 - val_acc: 0.6258\n",
            "\n",
            "Epoch 00052: val_acc did not improve from 0.62719\n",
            "Epoch 53/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 1.0482 - acc: 0.6064 - val_loss: 1.0171 - val_acc: 0.6225\n",
            "\n",
            "Epoch 00053: val_acc did not improve from 0.62719\n",
            "Epoch 54/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 1.0407 - acc: 0.6059 - val_loss: 1.0259 - val_acc: 0.6199\n",
            "\n",
            "Epoch 00054: val_acc did not improve from 0.62719\n",
            "Epoch 55/150\n",
            "248/248 [==============================] - 30s 119ms/step - loss: 1.0451 - acc: 0.6088 - val_loss: 1.0161 - val_acc: 0.6222\n",
            "\n",
            "Epoch 00055: val_acc did not improve from 0.62719\n",
            "Epoch 56/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 1.0423 - acc: 0.6109 - val_loss: 1.0054 - val_acc: 0.6264\n",
            "\n",
            "Epoch 00056: val_acc did not improve from 0.62719\n",
            "Epoch 57/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 1.0338 - acc: 0.6150 - val_loss: 1.0114 - val_acc: 0.6239\n",
            "\n",
            "Epoch 00057: val_acc did not improve from 0.62719\n",
            "Epoch 58/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 1.0339 - acc: 0.6096 - val_loss: 1.0043 - val_acc: 0.6239\n",
            "\n",
            "Epoch 00058: val_acc did not improve from 0.62719\n",
            "Epoch 59/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 1.0288 - acc: 0.6114 - val_loss: 1.0033 - val_acc: 0.6269\n",
            "\n",
            "Epoch 00059: val_acc did not improve from 0.62719\n",
            "Epoch 60/150\n",
            "248/248 [==============================] - 29s 116ms/step - loss: 1.0256 - acc: 0.6154 - val_loss: 1.0020 - val_acc: 0.6230\n",
            "\n",
            "Epoch 00060: val_acc did not improve from 0.62719\n",
            "Epoch 61/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 1.0275 - acc: 0.6144 - val_loss: 1.0111 - val_acc: 0.6291\n",
            "\n",
            "Epoch 00061: val_acc improved from 0.62719 to 0.62914, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 62/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 1.0192 - acc: 0.6168 - val_loss: 1.0003 - val_acc: 0.6336\n",
            "\n",
            "Epoch 00062: val_acc improved from 0.62914 to 0.63360, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 63/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 1.0185 - acc: 0.6158 - val_loss: 1.0015 - val_acc: 0.6280\n",
            "\n",
            "Epoch 00063: val_acc did not improve from 0.63360\n",
            "Epoch 64/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 1.0169 - acc: 0.6173 - val_loss: 1.0030 - val_acc: 0.6264\n",
            "\n",
            "Epoch 00064: val_acc did not improve from 0.63360\n",
            "Epoch 65/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 1.0197 - acc: 0.6153 - val_loss: 1.0046 - val_acc: 0.6266\n",
            "\n",
            "Epoch 00065: val_acc did not improve from 0.63360\n",
            "Epoch 66/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 1.0133 - acc: 0.6195 - val_loss: 0.9904 - val_acc: 0.6322\n",
            "\n",
            "Epoch 00066: val_acc did not improve from 0.63360\n",
            "Epoch 67/150\n",
            "248/248 [==============================] - 29s 119ms/step - loss: 1.0166 - acc: 0.6180 - val_loss: 0.9999 - val_acc: 0.6305\n",
            "\n",
            "Epoch 00067: val_acc did not improve from 0.63360\n",
            "Epoch 68/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 1.0082 - acc: 0.6238 - val_loss: 0.9888 - val_acc: 0.6361\n",
            "\n",
            "Epoch 00068: val_acc improved from 0.63360 to 0.63611, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 69/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 1.0067 - acc: 0.6189 - val_loss: 0.9987 - val_acc: 0.6303\n",
            "\n",
            "Epoch 00069: val_acc did not improve from 0.63611\n",
            "Epoch 70/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 1.0084 - acc: 0.6244 - val_loss: 0.9998 - val_acc: 0.6269\n",
            "\n",
            "Epoch 00070: val_acc did not improve from 0.63611\n",
            "Epoch 71/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 1.0031 - acc: 0.6263 - val_loss: 0.9892 - val_acc: 0.6356\n",
            "\n",
            "Epoch 00071: val_acc did not improve from 0.63611\n",
            "Epoch 72/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 0.9945 - acc: 0.6230 - val_loss: 0.9985 - val_acc: 0.6291\n",
            "\n",
            "Epoch 00072: val_acc did not improve from 0.63611\n",
            "Epoch 73/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 0.9978 - acc: 0.6248 - val_loss: 1.0016 - val_acc: 0.6303\n",
            "\n",
            "Epoch 00073: val_acc did not improve from 0.63611\n",
            "Epoch 74/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 0.9960 - acc: 0.6289 - val_loss: 0.9999 - val_acc: 0.6303\n",
            "\n",
            "Epoch 00074: val_acc did not improve from 0.63611\n",
            "Epoch 75/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 0.9939 - acc: 0.6307 - val_loss: 0.9989 - val_acc: 0.6369\n",
            "\n",
            "Epoch 00075: val_acc improved from 0.63611 to 0.63695, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 76/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 0.9873 - acc: 0.6313 - val_loss: 0.9929 - val_acc: 0.6400\n",
            "\n",
            "Epoch 00076: val_acc improved from 0.63695 to 0.64001, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 77/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 0.9907 - acc: 0.6307 - val_loss: 0.9865 - val_acc: 0.6353\n",
            "\n",
            "Epoch 00077: val_acc did not improve from 0.64001\n",
            "Epoch 78/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 0.9895 - acc: 0.6281 - val_loss: 0.9950 - val_acc: 0.6378\n",
            "\n",
            "Epoch 00078: val_acc did not improve from 0.64001\n",
            "Epoch 79/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 0.9873 - acc: 0.6269 - val_loss: 0.9910 - val_acc: 0.6383\n",
            "\n",
            "Epoch 00079: val_acc did not improve from 0.64001\n",
            "Epoch 80/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 0.9889 - acc: 0.6294 - val_loss: 0.9847 - val_acc: 0.6428\n",
            "\n",
            "Epoch 00080: val_acc improved from 0.64001 to 0.64280, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 81/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 0.9852 - acc: 0.6302 - val_loss: 0.9870 - val_acc: 0.6411\n",
            "\n",
            "Epoch 00081: val_acc did not improve from 0.64280\n",
            "Epoch 82/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 0.9863 - acc: 0.6293 - val_loss: 0.9962 - val_acc: 0.6322\n",
            "\n",
            "Epoch 00082: val_acc did not improve from 0.64280\n",
            "Epoch 83/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 0.9724 - acc: 0.6326 - val_loss: 0.9848 - val_acc: 0.6358\n",
            "\n",
            "Epoch 00083: val_acc did not improve from 0.64280\n",
            "Epoch 84/150\n",
            "248/248 [==============================] - 29s 116ms/step - loss: 0.9813 - acc: 0.6312 - val_loss: 0.9906 - val_acc: 0.6369\n",
            "\n",
            "Epoch 00084: val_acc did not improve from 0.64280\n",
            "Epoch 85/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 0.9707 - acc: 0.6391 - val_loss: 0.9929 - val_acc: 0.6389\n",
            "\n",
            "Epoch 00085: val_acc did not improve from 0.64280\n",
            "Epoch 86/150\n",
            "248/248 [==============================] - 29s 118ms/step - loss: 0.9779 - acc: 0.6352 - val_loss: 0.9903 - val_acc: 0.6381\n",
            "\n",
            "Epoch 00086: val_acc did not improve from 0.64280\n",
            "Epoch 87/150\n",
            "248/248 [==============================] - 29s 119ms/step - loss: 0.9761 - acc: 0.6362 - val_loss: 0.9942 - val_acc: 0.6375\n",
            "\n",
            "Epoch 00087: val_acc did not improve from 0.64280\n",
            "Epoch 88/150\n",
            "248/248 [==============================] - 29s 117ms/step - loss: 0.9726 - acc: 0.6325 - val_loss: 0.9883 - val_acc: 0.6386\n",
            "\n",
            "Epoch 00088: val_acc did not improve from 0.64280\n",
            "Epoch 89/150\n",
            "248/248 [==============================] - 30s 119ms/step - loss: 0.9709 - acc: 0.6362 - val_loss: 0.9841 - val_acc: 0.6417\n",
            "\n",
            "Epoch 00089: val_acc did not improve from 0.64280\n",
            "Epoch 90/150\n",
            "248/248 [==============================] - 30s 119ms/step - loss: 0.9715 - acc: 0.6344 - val_loss: 0.9848 - val_acc: 0.6445\n",
            "\n",
            "Epoch 00090: val_acc improved from 0.64280 to 0.64447, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 91/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 0.9693 - acc: 0.6344 - val_loss: 0.9875 - val_acc: 0.6417\n",
            "\n",
            "Epoch 00091: val_acc did not improve from 0.64447\n",
            "Epoch 92/150\n",
            "248/248 [==============================] - 30s 119ms/step - loss: 0.9643 - acc: 0.6358 - val_loss: 0.9938 - val_acc: 0.6411\n",
            "\n",
            "Epoch 00092: val_acc did not improve from 0.64447\n",
            "Epoch 93/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 0.9610 - acc: 0.6364 - val_loss: 0.9913 - val_acc: 0.6414\n",
            "\n",
            "Epoch 00093: val_acc did not improve from 0.64447\n",
            "Epoch 94/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 0.9649 - acc: 0.6386 - val_loss: 0.9786 - val_acc: 0.6459\n",
            "\n",
            "Epoch 00094: val_acc improved from 0.64447 to 0.64586, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 95/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 0.9612 - acc: 0.6403 - val_loss: 0.9782 - val_acc: 0.6434\n",
            "\n",
            "Epoch 00095: val_acc did not improve from 0.64586\n",
            "Epoch 96/150\n",
            "248/248 [==============================] - 30s 119ms/step - loss: 0.9595 - acc: 0.6386 - val_loss: 0.9816 - val_acc: 0.6408\n",
            "\n",
            "Epoch 00096: val_acc did not improve from 0.64586\n",
            "Epoch 97/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9563 - acc: 0.6411 - val_loss: 0.9840 - val_acc: 0.6417\n",
            "\n",
            "Epoch 00097: val_acc did not improve from 0.64586\n",
            "Epoch 98/150\n",
            "248/248 [==============================] - 31s 124ms/step - loss: 0.9578 - acc: 0.6430 - val_loss: 0.9858 - val_acc: 0.6453\n",
            "\n",
            "Epoch 00098: val_acc did not improve from 0.64586\n",
            "Epoch 99/150\n",
            "248/248 [==============================] - 31s 126ms/step - loss: 0.9612 - acc: 0.6402 - val_loss: 0.9809 - val_acc: 0.6425\n",
            "\n",
            "Epoch 00099: val_acc did not improve from 0.64586\n",
            "Epoch 100/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9539 - acc: 0.6407 - val_loss: 0.9865 - val_acc: 0.6414\n",
            "\n",
            "Epoch 00100: val_acc did not improve from 0.64586\n",
            "Epoch 101/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 0.9536 - acc: 0.6424 - val_loss: 0.9854 - val_acc: 0.6378\n",
            "\n",
            "Epoch 00101: val_acc did not improve from 0.64586\n",
            "Epoch 102/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9466 - acc: 0.6450 - val_loss: 0.9772 - val_acc: 0.6436\n",
            "\n",
            "Epoch 00102: val_acc did not improve from 0.64586\n",
            "Epoch 103/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9495 - acc: 0.6438 - val_loss: 0.9835 - val_acc: 0.6420\n",
            "\n",
            "Epoch 00103: val_acc did not improve from 0.64586\n",
            "Epoch 104/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 0.9492 - acc: 0.6456 - val_loss: 0.9846 - val_acc: 0.6450\n",
            "\n",
            "Epoch 00104: ReduceLROnPlateau reducing learning rate to 0.004999999888241291.\n",
            "\n",
            "Epoch 00104: val_acc did not improve from 0.64586\n",
            "Epoch 105/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 0.9370 - acc: 0.6509 - val_loss: 0.9768 - val_acc: 0.6475\n",
            "\n",
            "Epoch 00105: val_acc improved from 0.64586 to 0.64753, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 106/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9365 - acc: 0.6506 - val_loss: 0.9759 - val_acc: 0.6428\n",
            "\n",
            "Epoch 00106: val_acc did not improve from 0.64753\n",
            "Epoch 107/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9291 - acc: 0.6511 - val_loss: 0.9783 - val_acc: 0.6414\n",
            "\n",
            "Epoch 00107: val_acc did not improve from 0.64753\n",
            "Epoch 108/150\n",
            "248/248 [==============================] - 30s 123ms/step - loss: 0.9358 - acc: 0.6504 - val_loss: 0.9753 - val_acc: 0.6431\n",
            "\n",
            "Epoch 00108: val_acc did not improve from 0.64753\n",
            "Epoch 109/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9295 - acc: 0.6541 - val_loss: 0.9776 - val_acc: 0.6411\n",
            "\n",
            "Epoch 00109: val_acc did not improve from 0.64753\n",
            "Epoch 110/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9260 - acc: 0.6537 - val_loss: 0.9770 - val_acc: 0.6420\n",
            "\n",
            "Epoch 00110: val_acc did not improve from 0.64753\n",
            "Epoch 111/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9273 - acc: 0.6551 - val_loss: 0.9781 - val_acc: 0.6445\n",
            "\n",
            "Epoch 00111: val_acc did not improve from 0.64753\n",
            "Epoch 112/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 0.9261 - acc: 0.6540 - val_loss: 0.9777 - val_acc: 0.6484\n",
            "\n",
            "Epoch 00112: val_acc improved from 0.64753 to 0.64837, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 113/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 0.9262 - acc: 0.6531 - val_loss: 0.9755 - val_acc: 0.6456\n",
            "\n",
            "Epoch 00113: val_acc did not improve from 0.64837\n",
            "Epoch 114/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9199 - acc: 0.6572 - val_loss: 0.9738 - val_acc: 0.6481\n",
            "\n",
            "Epoch 00114: val_acc did not improve from 0.64837\n",
            "Epoch 115/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9238 - acc: 0.6545 - val_loss: 0.9703 - val_acc: 0.6500\n",
            "\n",
            "Epoch 00115: val_acc improved from 0.64837 to 0.65004, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 116/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9249 - acc: 0.6552 - val_loss: 0.9731 - val_acc: 0.6428\n",
            "\n",
            "Epoch 00116: val_acc did not improve from 0.65004\n",
            "Epoch 117/150\n",
            "248/248 [==============================] - 30s 120ms/step - loss: 0.9214 - acc: 0.6544 - val_loss: 0.9702 - val_acc: 0.6428\n",
            "\n",
            "Epoch 00117: val_acc did not improve from 0.65004\n",
            "Epoch 118/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 0.9267 - acc: 0.6548 - val_loss: 0.9715 - val_acc: 0.6470\n",
            "\n",
            "Epoch 00118: val_acc did not improve from 0.65004\n",
            "Epoch 119/150\n",
            "248/248 [==============================] - 30s 121ms/step - loss: 0.9236 - acc: 0.6531 - val_loss: 0.9670 - val_acc: 0.6459\n",
            "\n",
            "Epoch 00119: val_acc did not improve from 0.65004\n",
            "Epoch 120/150\n",
            "248/248 [==============================] - 33s 132ms/step - loss: 0.9226 - acc: 0.6545 - val_loss: 0.9663 - val_acc: 0.6537\n",
            "\n",
            "Epoch 00120: val_acc improved from 0.65004 to 0.65366, saving model to /content/drive/My Drive/cs230 project/models/Baseline-weights-best.hdf5\n",
            "Epoch 121/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 0.9150 - acc: 0.6577 - val_loss: 0.9660 - val_acc: 0.6467\n",
            "\n",
            "Epoch 00121: val_acc did not improve from 0.65366\n",
            "Epoch 122/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 0.9138 - acc: 0.6586 - val_loss: 0.9705 - val_acc: 0.6459\n",
            "\n",
            "Epoch 00122: val_acc did not improve from 0.65366\n",
            "Epoch 123/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 0.9176 - acc: 0.6568 - val_loss: 0.9713 - val_acc: 0.6431\n",
            "\n",
            "Epoch 00123: val_acc did not improve from 0.65366\n",
            "Epoch 124/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 0.9125 - acc: 0.6588 - val_loss: 0.9714 - val_acc: 0.6470\n",
            "\n",
            "Epoch 00124: val_acc did not improve from 0.65366\n",
            "Epoch 125/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 0.9144 - acc: 0.6595 - val_loss: 0.9734 - val_acc: 0.6442\n",
            "\n",
            "Epoch 00125: val_acc did not improve from 0.65366\n",
            "Epoch 126/150\n",
            "248/248 [==============================] - 30s 122ms/step - loss: 0.9164 - acc: 0.6588 - val_loss: 0.9700 - val_acc: 0.6475\n",
            "\n",
            "Epoch 00126: val_acc did not improve from 0.65366\n",
            "Epoch 127/150\n",
            "248/248 [==============================] - 31s 125ms/step - loss: 0.9174 - acc: 0.6579 - val_loss: 0.9674 - val_acc: 0.6470\n",
            "\n",
            "Epoch 00127: val_acc did not improve from 0.65366\n",
            "Epoch 128/150\n",
            "248/248 [==============================] - 32s 131ms/step - loss: 0.9161 - acc: 0.6587 - val_loss: 0.9646 - val_acc: 0.6481\n",
            "\n",
            "Epoch 00128: val_acc did not improve from 0.65366\n",
            "Epoch 129/150\n",
            "248/248 [==============================] - 32s 130ms/step - loss: 0.9146 - acc: 0.6563 - val_loss: 0.9692 - val_acc: 0.6503\n",
            "\n",
            "Epoch 00129: val_acc did not improve from 0.65366\n",
            "Epoch 130/150\n",
            "248/248 [==============================] - 32s 129ms/step - loss: 0.9175 - acc: 0.6569 - val_loss: 0.9710 - val_acc: 0.6453\n",
            "\n",
            "Epoch 00130: ReduceLROnPlateau reducing learning rate to 0.0024999999441206455.\n",
            "\n",
            "Epoch 00130: val_acc did not improve from 0.65366\n",
            "Epoch 131/150\n",
            "248/248 [==============================] - 32s 130ms/step - loss: 0.9053 - acc: 0.6624 - val_loss: 0.9674 - val_acc: 0.6453\n",
            "\n",
            "Epoch 00131: val_acc did not improve from 0.65366\n",
            "Epoch 132/150\n",
            "248/248 [==============================] - 31s 125ms/step - loss: 0.9101 - acc: 0.6579 - val_loss: 0.9695 - val_acc: 0.6450\n",
            "\n",
            "Epoch 00132: val_acc did not improve from 0.65366\n",
            "Epoch 133/150\n",
            "248/248 [==============================] - 31s 123ms/step - loss: 0.9078 - acc: 0.6596 - val_loss: 0.9670 - val_acc: 0.6447\n",
            "\n",
            "Epoch 00133: val_acc did not improve from 0.65366\n",
            "Epoch 134/150\n",
            "248/248 [==============================] - 31s 123ms/step - loss: 0.9038 - acc: 0.6622 - val_loss: 0.9661 - val_acc: 0.6509\n",
            "\n",
            "Epoch 00134: val_acc did not improve from 0.65366\n",
            "Epoch 135/150\n",
            "248/248 [==============================] - 30s 123ms/step - loss: 0.9074 - acc: 0.6584 - val_loss: 0.9648 - val_acc: 0.6459\n",
            "\n",
            "Epoch 00135: val_acc did not improve from 0.65366\n",
            "Epoch 136/150\n",
            "248/248 [==============================] - 31s 123ms/step - loss: 0.9087 - acc: 0.6588 - val_loss: 0.9670 - val_acc: 0.6473\n",
            "\n",
            "Epoch 00136: val_acc did not improve from 0.65366\n",
            "Epoch 137/150\n",
            "248/248 [==============================] - 30s 123ms/step - loss: 0.9068 - acc: 0.6617 - val_loss: 0.9666 - val_acc: 0.6503\n",
            "\n",
            "Epoch 00137: val_acc did not improve from 0.65366\n",
            "Epoch 138/150\n",
            "248/248 [==============================] - 30s 123ms/step - loss: 0.9061 - acc: 0.6587 - val_loss: 0.9658 - val_acc: 0.6509\n",
            "\n",
            "Epoch 00138: val_acc did not improve from 0.65366\n",
            "Epoch 139/150\n",
            "110/248 [============>.................] - ETA: 16s - loss: 0.9045 - acc: 0.6637"
          ],
          "name": "stdout"
        }
      ]
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "5aUXi05NXeol",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print('\\n# Evaluate on dev data')\n",
        "results_dev = model.evaluate_generator(dev_generator, 3509 // BS)\n",
        "print('dev loss, dev acc:', results_dev)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "iXTV7TokXfx1",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print('\\n# Evaluate on test data')\n",
        "results_test = model.evaluate_generator(test_generator, 3509 // BS)\n",
        "print('test loss, test acc:', results_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "N65tgp91XiWO",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "# list all data in history\n",
        "print(history.history.keys())\n",
        "# summarize history for accuracy\n",
        "plt.plot(history.history['acc'])\n",
        "plt.plot(history.history['val_acc'])\n",
        "plt.title('model accuracy')\n",
        "plt.ylabel('accuracy')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'dev'], loc='upper left')\n",
        "plt.show()\n",
        "# summarize history for loss\n",
        "plt.plot(history.history['loss'])\n",
        "plt.plot(history.history['val_loss'])\n",
        "plt.title('model loss')\n",
        "plt.ylabel('loss')\n",
        "plt.xlabel('epoch')\n",
        "plt.legend(['train', 'dev'], loc='upper left')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "CppmEt5KXlYz",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "epoch_str = '-EPOCHS_' + str(EPOCHS)\n",
        "dropout_str = '-DROPOUT_' + str(DROPOUT_RATE)\n",
        "test_acc = '-test_acc_%.3f' % results_test[1]\n",
        "model.save('/content/drive/My Drive/cs230 project/models/' + 'SOA' + epoch_str + dropout_str + test_acc + '.h5')"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "27rHaOQlzzCY",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "emotions = {0:'Angry', 1:'Disgust', 2:'Fear', 3:'Happy', 4:'Sad', 5:'Surprise', 6:'Neutral'}\n",
        "\n",
        "y_pred = model.predict_generator(dev_generator).argmax(axis=1)\n",
        "y_true = dev_generator.classes\n",
        "\n",
        "cmat_df_test=pd.DataFrame(\n",
        "  confusion_matrix(y_true, y_pred, normalize='true').round(2),\n",
        "  index=emotions.values(), \n",
        "  columns=emotions.values()\n",
        "  )\n",
        "\n",
        "plt.figure(figsize=(5,5))\n",
        "heatmap(cmat_df_test,annot=True,cmap=plt.cm.Blues)\n",
        "plt.tight_layout()\n",
        "plt.title('Confusion Matrix on Private Test Set')\n",
        "plt.ylabel('True label')\n",
        "plt.xlabel('Predicted label')\n",
        "plt.show()"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "ZMAJ9smqKdD3",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "from sklearn.metrics import accuracy_score\n",
        "# configure image data augmentation\n",
        "datagen = ImageDataGenerator(horizontal_flip=True)\n",
        "\n",
        "# make a prediction using test-time augmentation\n",
        "def tta_prediction(datagen, model, image, n_examples):\n",
        "\t# convert image into dataset\n",
        "\tsamples = np.expand_dims(image, 0)\n",
        "\t# prepare iterator\n",
        "\tit = datagen.flow(samples, batch_size=n_examples)\n",
        "\t# make predictions for each augmented image\n",
        "\tyhats = model.predict_generator(it, steps=n_examples, verbose=0)\n",
        "\t# sum across predictions\n",
        "\tsummed = np.sum(yhats, axis=0)\n",
        "\t# argmax across classes\n",
        "\treturn np.argmax(summed)\n",
        " \n",
        " # evaluate a model on a dataset using test-time augmentation\n",
        "def tta_evaluate_model(model, testX, testY):\n",
        "\t# configure image data augmentation\n",
        "\tdatagen = ImageDataGenerator(horizontal_flip=True)\n",
        "\t# define the number of augmented images to generate per test set image\n",
        "\tn_examples_per_image = 7\n",
        "\tyhats = list()\n",
        "\tfor i in range(len(testX)):\n",
        "\t\t# make augmented prediction\n",
        "\t\tyhat = tta_prediction(datagen, model, testX[i], n_examples_per_image)\n",
        "\t\t# store for evaluation\n",
        "\t\tyhats.append(yhat)\n",
        "\t# calculate accuracy\n",
        "\ttestY_labels = np.argmax(testY, axis=1)\n",
        "\tacc = accuracy_score(testY_labels, yhats)\n",
        "\treturn acc"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "gfTihcArMdUk",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        "print('\\n# Evaluate on test data')\n",
        "#TTA_results_test = tta_evaluate_model(model, X_test, Y_test)\n",
        "print('test loss, test acc:', results_test)\n",
        "print('TTA test acc:', TTA_results_test)"
      ],
      "execution_count": 0,
      "outputs": []
    },
    {
      "cell_type": "code",
      "metadata": {
        "id": "V57Y-EomzHtL",
        "colab_type": "code",
        "colab": {}
      },
      "source": [
        ""
      ],
      "execution_count": 0,
      "outputs": []
    }
  ]
}